#ifndef _PHRASER_H_
#define _PHRASER_H_
#include <iostream>
#include <fstream>
#include <algorithm>
#include <vector>
#include <boost/functional/hash.hpp>
#include <google/sparse_hash_map>
#include <boost/program_options.hpp>
#include <re2/re2.h>
#include <stdio.h>
#include <string>
#include <stdlib.h>
#include "client/dbclient.h"
#include "ztexting.h"
#include <iostream>

using namespace std;
using namespace mongo;
using namespace re2;

typedef google::sparse_hash_map<string, int, boost::hash<string> > Dict;
typedef Dict::value_type Item;

namespace po = boost::program_options;
struct _word_count_config:zxlib::zconfig_yaml{

     string host;
     int port;
     string field;
     string collection;
     string filter_file;

     BSONObj query_obj;
     void set_config(const YAML::Node& node )
     {
          node["host"]>> host;
          node["port"]>> port;
          node["field"]>> field;
          node["collection"]>> collection;
          node["filter_file"]>> filter_file;

          BSONObjBuilder builder;
          const YAML::Node& query = node["query"];
          for(YAML::Iterator it=query.begin();it!=query.end();++it) {
               std::string key, string_value;
               int int_value;
               it.first() >> key;
               if( key.compare("page_type") == 0 ){
                    it.second() >> int_value;
                    builder.append(key,int_value);
                    std::cout << "Key: " << key << ", value: " << int_value << std::endl;
               } else {
                    it.second() >> string_value;
                    builder.append(key,string_value);
                    std::cout << "Key: " << key << ", value: " << string_value << std::endl;
               }
          }
          query_obj = builder.obj();
     }
     void inspect()
     {
          cout << "host:" << host << endl;
          cout << "port:" << port << endl;
          cout << "field:" << field << endl;
          cout << "collection:" << collection << endl;
          cout << "filter_file:" << filter_file << endl;
     }
};
typedef struct _word_count_config word_count_config;
struct cmd_line { 
     cmd_line() : limit(0),one_column(false) { } 

     enum { 
          DefaultDBPort = 27017,
     };

     int limit;
     bool one_column;
     string output_file;
     string config_file;
     string ztexting_config_file;
};

typedef google::sparse_hash_map<string, int, boost::hash<string> > Dict;
typedef Dict::value_type Item;
namespace zxlib{
     class word_filter{
          private:
               vector<string> m_filter_words;
               string  m_filter_pattern;
          public:
               void load_filter(const char* filter_file){
                    zxlib::readlines(filter_file, m_filter_words);
                    m_filter_pattern+=zxlib::join(m_filter_words,"|");
               }
               word_filter( const char* filter_file ){
                    load_filter( filter_file );
               }
               word_filter(){ }
               string& get_pattern(){ return  m_filter_pattern;}
               vector<string>& get_filter_words(){
                    return m_filter_words;
               }
     }; //word_filter

     class phraser{
          private:
               cmd_line m_cmd_line;
               word_count_config m_config;
               word_filter m_filter;
               DBClientConnection m_connection;
               string m_connection_string;
               Dict m_word_map;
          public:
               phraser(cmd_line& cmd_line);
               void by_length();
               void by_stats();
               void prepare_cws();
               void save_to_file(Dict& words , const char* filename);
               bool filter_word( const char* word );
               bool stats( term_array& terms ,const char* org_text );
     };
}
#endif
